﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Collections;
using System.Threading;


namespace _1.BaiduSypider
{
    public partial class Form1 : Form
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private int failCount = 0;
        public Form1()
        {
            InitializeComponent();
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }

        private void textBoxKeyWord_TextChanged(object sender, EventArgs e)
        {

        }

        private void buttonGetResult_Click(object sender, EventArgs e)
        {
            string keyWord = textBoxKeyWord.Text;
            byte[] data = Encoding.UTF8.GetBytes(keyWord);
            string htmlKeyWord = BitConverter.ToString(data).Replace("-", "%");
            
            //string url = string.Format("https://www.baidu.com/s?tn=02003390_71_hao_pg&ie=utf-8&wd=%{0}", htmlKeyWord);
            string url = string.Format("https://www.baidu.com/s?rtt=1&bsst=1&cl=2&tn=news&word=%{0}&fr=zhidao", htmlKeyWord);

            urls[url] = false;
            Crawl(url);

            
            while(true)
            {
                string url1 = null, url2 = null;
                foreach (string turl in urls.Keys)
                {
                    if ((bool)urls[turl]) continue;
                    url1 = turl;
                    urls[url1] = true;
                    break;
                }

              
                foreach (string turl in urls.Keys)
                {
                    if ((bool)urls[turl]) continue;
                    url2 = turl;
                    urls[url2] = true;
                    break;
                }

                if (url1 == null || url2 == null ) break;
                
                Task<string>[] tasks =
                {
                    Task.Run(()=>Crawl(url1)),
                    Task.Run(()=>Crawl(url2)),

                };
                textBox1.Text += tasks[0].Result + "\r\n";
                textBox2.Text += tasks[1].Result + "\r\n";
            }
            
        }

        public string SubstringSingle(string source)
        {
            //Regex rg = new Regex(@"<p>[.*?]</p>", RegexOptions.Multiline | RegexOptions.Singleline);
            //return rg.Match(source).Value;
            string restult = "";
            foreach (Match m in Regex.Matches(source, @"(?ins)(?<=(</a>\s*<(?<mark>p[^>]*>)|<(?<mark>span)[^>]*>))[\s\S]+?(?=</\k<mark>)"))
            {
                restult += m.Value;
            }
            return restult;

        }



        private string Crawl(string current)
        {
            //Console.WriteLine("开始爬行了.....");
        
            //https://www.dy2018.com/1/?jdfwkey=sd60d
            //string current = null;
            //foreach (string url in urls.Keys)
            //{
            //    if ((bool)urls[url]) continue;
            //    current = url;
            //}
            

            Console.WriteLine("爬行" + current + "页面!");

            string html = DownLoad(current);
                
            urls[current] = true;
            count++;

            if (count == 1)
            {
                //Console.WriteLine(html);
                Parse(html, current);//只解析首页的html，只爬取起始网站上的网页
                return "";
            }
            else
            {
                string s = SubstringSingle(html);
                if (s.Length < 200) return s;
                else return s.Substring(0, 200);
            }
            
            //return "";
            
        }

        private bool isLuan(string txt)
        {
            var bytes = Encoding.UTF8.GetBytes(txt);
            //239 191 189
            for (var i = 0; i < bytes.Length; i++)
            {
                if (i < bytes.Length - 3)
                    if (bytes[i] == 239 && bytes[i + 1] == 191 && bytes[i + 2] == 189)
                    {
                        return true;
                    }
            }
            return false;
        }

        private string DownLoad(string url)
        {
            try
            {

                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
                //设置请求方法
                request.Method = "GET";
                //设置请求头
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36";
                //获取resonse
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                //获取结果转化为string
                string html;

                string characterSet = response.CharacterSet;
                System.Text.Encoding encode;
                //Encoding encode;
                if (characterSet != "")
                {
                    if (characterSet == "ISO-8859-1")
                    {
                        characterSet = "gb2312";
                    }
                    //textBox1.Text += characterSet + "\t";
                    encode = Encoding.GetEncoding(characterSet);
                }
                else
                {
                    encode = Encoding.Default;
                }

                using (Stream stream = response.GetResponseStream())
                {
                    using (StreamReader reader = new StreamReader(stream, encode))
                    {
                        html = reader.ReadToEnd();
                    }

                }

             
                //System.Console.WriteLine(url);
                
                
                //textBox1.Text += url + "\r\n";
                string fileName = count.ToString() + ".txt";
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                
                failCount += 1;
                return "";
            }
        }


        public void Parse(string html, string currentUrl)
        {
            //string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
       
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
               
                if (strRef.IndexOf(".ico") >= 0 || strRef.IndexOf(".css") >= 0 || strRef.IndexOf(".svg") >= 0) continue;
                //if (strRef.IndexOf(currentUrl) < 0) strRef = currentUrl + strRef;
                if (strRef.IndexOf("http") < 0) strRef = "https:" + strRef;
                if (strRef.Length == 0 || strRef.IndexOf("baidu") >= 0 || strRef.Length > 100) continue;
                Console.WriteLine(strRef);
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }

    }
}
